

<!DOCTYPE html>
<html lang="zh-CN" data-default-color-scheme=auto>



<head>
  <meta charset="UTF-8">
  <link rel="apple-touch-icon" sizes="76x76" href="/img/fluid.png">
  <link rel="icon" href="/img/fluid.png">
  <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=5.0, shrink-to-fit=no">
  <meta http-equiv="x-ua-compatible" content="ie=edge">
  
  <meta name="theme-color" content="#2f4154">
  <meta name="author" content="John Doe">
  <meta name="keywords" content="">
  
    <meta name="description" content="运行环境  windows10 anaconda3 ( python3都行) pycharm 社区版 谷歌浏览器 ( 其他的也行 )  本次Demo用到的库及其作用如下：  requests库 模拟用户提交 HTTP请求，获取网页的内容 re库 对字符串进行正则表达式匹配，用于获取目标内容 bs4库，对网页字符串进行格式化，可通过类名、ID、标签名三种方式定义对应的标签 pandas库，将爬取的内">
<meta property="og:type" content="article">
<meta property="og:title" content="Python爬虫 笔记1 _ 爬虫案例及模板 简要四步爬取购物网站信息 _ 基于 requests、re、 pandas、 bs4">
<meta property="og:url" content="http://example.com/2021/12/08/Python%E7%88%AC%E8%99%AB%20%E7%AC%94%E8%AE%B01%20_%20%E7%88%AC%E8%99%AB%E6%A1%88%E4%BE%8B%E5%8F%8A%E6%A8%A1%E6%9D%BF%20%E7%AE%80%E8%A6%81%E5%9B%9B%E6%AD%A5%E7%88%AC%E5%8F%96%E8%B4%AD%E7%89%A9%E7%BD%91%E7%AB%99%E4%BF%A1%E6%81%AF%20_%20%E5%9F%BA%E4%BA%8E%20requests%E3%80%81re%E3%80%81%20pandas%E3%80%81%20bs4/index.html">
<meta property="og:site_name" content="Hexo">
<meta property="og:description" content="运行环境  windows10 anaconda3 ( python3都行) pycharm 社区版 谷歌浏览器 ( 其他的也行 )  本次Demo用到的库及其作用如下：  requests库 模拟用户提交 HTTP请求，获取网页的内容 re库 对字符串进行正则表达式匹配，用于获取目标内容 bs4库，对网页字符串进行格式化，可通过类名、ID、标签名三种方式定义对应的标签 pandas库，将爬取的内">
<meta property="og:locale" content="zh_CN">
<meta property="og:image" content="https://img-blog.csdnimg.cn/e5aecfb20b4d44c994d1934082148556.png">
<meta property="og:image" content="https://img-blog.csdnimg.cn/26f5bb23926e44f282c241ad17286530.png">
<meta property="og:image" content="https://img-blog.csdnimg.cn/ca6597f3a10b47eaaeeacc1753595336.png">
<meta property="og:image" content="https://img-blog.csdnimg.cn/ea9ecabd0c0541609be8ef93e8777f92.png">
<meta property="og:image" content="https://img-blog.csdnimg.cn/b73c0458612b40b78d319d6d9625dcd5.png">
<meta property="og:image" content="https://img-blog.csdnimg.cn/8ef29ef326df409a84c806df7a4f6168.png">
<meta property="og:image" content="https://img-blog.csdnimg.cn/528ff4cb9acc4d1f826de42b8e1d233b.png">
<meta property="og:image" content="https://img-blog.csdnimg.cn/eaf5a7d1369c4e2fac300dcd6de59b10.png">
<meta property="og:image" content="https://img-blog.csdnimg.cn/b164162647c447a68daf03cb3736e187.png">
<meta property="article:published_time" content="2021-12-07T16:12:16.000Z">
<meta property="article:modified_time" content="2022-08-22T15:47:38.224Z">
<meta property="article:author" content="John Doe">
<meta property="article:tag" content="Python">
<meta property="article:tag" content="爬虫">
<meta name="twitter:card" content="summary_large_image">
<meta name="twitter:image" content="https://img-blog.csdnimg.cn/e5aecfb20b4d44c994d1934082148556.png">
  
  
  
  <title>Python爬虫 笔记1 _ 爬虫案例及模板 简要四步爬取购物网站信息 _ 基于 requests、re、 pandas、 bs4 - Hexo</title>

  <link  rel="stylesheet" href="https://lib.baomitu.com/twitter-bootstrap/4.6.1/css/bootstrap.min.css" />



  <link  rel="stylesheet" href="https://lib.baomitu.com/github-markdown-css/4.0.0/github-markdown.min.css" />

  <link  rel="stylesheet" href="https://lib.baomitu.com/hint.css/2.7.0/hint.min.css" />

  <link  rel="stylesheet" href="https://lib.baomitu.com/fancybox/3.5.7/jquery.fancybox.min.css" />



<!-- 主题依赖的图标库，不要自行修改 -->
<!-- Do not modify the link that theme dependent icons -->

<link rel="stylesheet" href="//at.alicdn.com/t/font_1749284_hj8rtnfg7um.css">



<link rel="stylesheet" href="//at.alicdn.com/t/font_1736178_lbnruvf0jn.css">


<link  rel="stylesheet" href="/css/main.css" />


  <link id="highlight-css" rel="stylesheet" href="/css/highlight.css" />
  
    <link id="highlight-css-dark" rel="stylesheet" href="/css/highlight-dark.css" />
  




  <script id="fluid-configs">
    var Fluid = window.Fluid || {};
    Fluid.ctx = Object.assign({}, Fluid.ctx)
    var CONFIG = {"hostname":"example.com","root":"/","version":"1.9.2","typing":{"enable":true,"typeSpeed":70,"cursorChar":"_","loop":false,"scope":[]},"anchorjs":{"enable":true,"element":"h1,h2,h3,h4,h5,h6","placement":"left","visible":"hover","icon":""},"progressbar":{"enable":true,"height_px":3,"color":"#29d","options":{"showSpinner":false,"trickleSpeed":100}},"code_language":{"enable":true,"default":"TEXT"},"copy_btn":true,"image_caption":{"enable":true},"image_zoom":{"enable":true,"img_url_replace":["",""]},"toc":{"enable":true,"placement":"right","headingSelector":"h1,h2,h3,h4,h5,h6","collapseDepth":0},"lazyload":{"enable":true,"loading_img":"/img/loading.gif","onlypost":false,"offset_factor":2},"web_analytics":{"enable":false,"follow_dnt":true,"baidu":null,"google":null,"gtag":null,"tencent":{"sid":null,"cid":null},"woyaola":null,"cnzz":null,"leancloud":{"app_id":null,"app_key":null,"server_url":null,"path":"window.location.pathname","ignore_local":false}},"search_path":"/local-search.xml"};

    if (CONFIG.web_analytics.follow_dnt) {
      var dntVal = navigator.doNotTrack || window.doNotTrack || navigator.msDoNotTrack;
      Fluid.ctx.dnt = dntVal && (dntVal.startsWith('1') || dntVal.startsWith('yes') || dntVal.startsWith('on'));
    }
  </script>
  <script  src="/js/utils.js" ></script>
  <script  src="/js/color-schema.js" ></script>
  


  
<meta name="generator" content="Hexo 6.2.0"></head>


<body>
  

  <header>
    

<div class="header-inner" style="height: 70vh;">
  <nav id="navbar" class="navbar fixed-top  navbar-expand-lg navbar-dark scrolling-navbar">
  <div class="container">
    <a class="navbar-brand" href="/">
      <strong>尤 Ni&#39;s Blog</strong>
    </a>

    <button id="navbar-toggler-btn" class="navbar-toggler" type="button" data-toggle="collapse"
            data-target="#navbarSupportedContent"
            aria-controls="navbarSupportedContent" aria-expanded="false" aria-label="Toggle navigation">
      <div class="animated-icon"><span></span><span></span><span></span></div>
    </button>

    <!-- Collapsible content -->
    <div class="collapse navbar-collapse" id="navbarSupportedContent">
      <ul class="navbar-nav ml-auto text-center">
        
          
          
          
          
            <li class="nav-item">
              <a class="nav-link" href="/">
                <i class="iconfont icon-home-fill"></i>
                首页
              </a>
            </li>
          
        
          
          
          
          
            <li class="nav-item">
              <a class="nav-link" href="/archives/">
                <i class="iconfont icon-archive-fill"></i>
                归档
              </a>
            </li>
          
        
          
          
          
          
            <li class="nav-item">
              <a class="nav-link" href="/categories/">
                <i class="iconfont icon-category-fill"></i>
                分类
              </a>
            </li>
          
        
          
          
          
          
            <li class="nav-item">
              <a class="nav-link" href="/tags/">
                <i class="iconfont icon-tags-fill"></i>
                标签
              </a>
            </li>
          
        
          
          
          
          
            <li class="nav-item">
              <a class="nav-link" href="/about/">
                <i class="iconfont icon-user-fill"></i>
                关于
              </a>
            </li>
          
        
        
          <li class="nav-item" id="search-btn">
            <a class="nav-link" target="_self" href="javascript:;" data-toggle="modal" data-target="#modalSearch" aria-label="Search">
              &nbsp;<i class="iconfont icon-search"></i>&nbsp;
            </a>
          </li>
          
        
        
          <li class="nav-item" id="color-toggle-btn">
            <a class="nav-link" target="_self" href="javascript:;" aria-label="Color Toggle">&nbsp;<i
                class="iconfont icon-dark" id="color-toggle-icon"></i>&nbsp;</a>
          </li>
        
      </ul>
    </div>
  </div>
</nav>

  

<div id="banner" class="banner" parallax=true
     style="background: url('/img/default.png') no-repeat center center; background-size: cover;">
  <div class="full-bg-img">
    <div class="mask flex-center" style="background-color: rgba(0, 0, 0, 0.3)">
      <div class="banner-text text-center fade-in-up">
        <div class="h2">
          
            <span id="subtitle" data-typed-text="Python爬虫 笔记1 _ 爬虫案例及模板 简要四步爬取购物网站信息 _ 基于 requests、re、 pandas、 bs4"></span>
          
        </div>

        
          
  <div class="mt-3">
    
      <span class="post-meta mr-2">
        <i class="iconfont icon-author" aria-hidden="true"></i>
        John Doe
      </span>
    
    
      <span class="post-meta">
        <i class="iconfont icon-date-fill" aria-hidden="true"></i>
        <time datetime="2021-12-08 00:12" pubdate>
          2021年12月8日 凌晨
        </time>
      </span>
    
  </div>

  <div class="mt-1">
    
      <span class="post-meta mr-2">
        <i class="iconfont icon-chart"></i>
        
          4.3k 字
        
      </span>
    

    
      <span class="post-meta mr-2">
        <i class="iconfont icon-clock-fill"></i>
        
        
        
          36 分钟
        
      </span>
    

    
    
  </div>


        
      </div>

      
    </div>
  </div>
</div>

</div>

  </header>

  <main>
    
      

<div class="container-fluid nopadding-x">
  <div class="row nomargin-x">
    <div class="side-col d-none d-lg-block col-lg-2">
      

    </div>

    <div class="col-lg-8 nopadding-x-md">
      <div class="container nopadding-x-md" id="board-ctn">
        <div id="board">
          <article class="post-content mx-auto">
            <!-- SEO header -->
            <h1 style="display: none">Python爬虫 笔记1 _ 爬虫案例及模板 简要四步爬取购物网站信息 _ 基于 requests、re、 pandas、 bs4</h1>
            
              <p class="note note-info">
                
                  
                    本文最后更新于：1 小时前
                  
                
              </p>
            
            
              <div class="markdown-body">
                
                <h1 id="运行环境"><a href="#运行环境" class="headerlink" title="运行环境"></a>运行环境</h1><hr>
<ul>
<li>windows10</li>
<li>anaconda3 ( python3都行)</li>
<li>pycharm 社区版</li>
<li>谷歌浏览器 ( 其他的也行 )</li>
</ul>
<p>本次Demo用到的库及其作用如下：</p>
<ul>
<li><strong>requests库</strong> 模拟用户提交 HTTP请求，获取网页的内容</li>
<li><strong>re库</strong> 对字符串进行正则表达式匹配，用于获取目标内容</li>
<li><strong>bs4库</strong>，对网页字符串进行格式化，可通过类名、ID、标签名三种方式定义对应的标签</li>
<li><strong>pandas库</strong>，将爬取的内容存入DataFrame二维表，方便查看与导出</li>
</ul>
<p>首先导入需要用到的库</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><code class="hljs python"><span class="hljs-keyword">import</span> requests<br><span class="hljs-keyword">import</span> re<br><span class="hljs-keyword">from</span> bs4 <span class="hljs-keyword">import</span> BeautifulSoup<br><span class="hljs-keyword">import</span> pandas <span class="hljs-keyword">as</span> pd<br></code></pre></td></tr></table></figure>
<p>提示：若在运行中报错，则可能是因为python环境中缺少对应的库，可以在python shell里安装</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><code class="hljs shell">pip install 库名<br><span class="hljs-meta prompt_">#</span><span class="language-bash">3.x版本</span><br>pip3 install 库名<br></code></pre></td></tr></table></figure>

<p>@[toc]</p>
<h1 id="爬虫模板"><a href="#爬虫模板" class="headerlink" title="爬虫模板"></a>爬虫模板</h1><hr>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br></pre></td><td class="code"><pre><code class="hljs python"><span class="hljs-comment"># 一、导库</span><br><span class="hljs-keyword">import</span> requests, re, pandas <span class="hljs-keyword">as</span> pd<br><span class="hljs-keyword">from</span> bs4 <span class="hljs-keyword">import</span> BeautifulSoup<br><br><span class="hljs-comment"># 二、设置请求网页的信息: 网址url + header请求头</span><br>url = <span class="hljs-string">&#x27;爬取的网站地址&#x27;</span><br>headers = &#123;<br>    <span class="hljs-string">&#x27;User-Agent&#x27;</span>:  <span class="hljs-string">&#x27;通过浏览器调试工具查看&#x27;</span>,<br>&#125;<br><br><span class="hljs-comment"># 三、使用 requests库 获取网页响应的结果文件</span><br>html = requests.get(url, headers=headers).text<br><br><span class="hljs-comment"># 四、使用 BeautifulSoup库 解析HTML网页内容</span><br>soup = BeautifulSoup(html)<br><br><span class="hljs-comment"># 五、结果bs库和re库的正则表达式提取需要的数据</span><br>data = []<br><span class="hljs-keyword">for</span> x <span class="hljs-keyword">in</span> soup.select(<span class="hljs-string">&quot;定位标签&quot;</span>):<br>    data.append(re.search(<span class="hljs-string">&#x27;&gt;([^&lt;]+)&lt;&#x27;</span>, <span class="hljs-built_in">str</span>(x)).group(<span class="hljs-number">1</span>))<br><br><span class="hljs-comment"># 六、将结果保存到 DataFrame对象，并将结果导出到本地文件</span><br>df = pd.DataFrame(&#123;<span class="hljs-string">&#x27;中国省份&#x27;</span> : data&#125;)<br>df.to_csv(<span class="hljs-string">&#x27;导出的文件名.txt&#x27;</span>, sep=<span class="hljs-string">&#x27;\t&#x27;</span>,index=<span class="hljs-literal">False</span>, header=<span class="hljs-literal">None</span>)<br></code></pre></td></tr></table></figure>
<h1 id="爬虫第一步：获取-URL，设置请求头User-Agent"><a href="#爬虫第一步：获取-URL，设置请求头User-Agent" class="headerlink" title="爬虫第一步：获取 URL，设置请求头User-Agent"></a>爬虫第一步：获取 URL，设置请求头User-Agent</h1><hr>
<p>现在许多网站都有反扒机制，比如笔者在尝试爬取京东购物网时，发现需要登录才能爬取，而且第一页和第二页的网址URL不仅仅是单个参数的变化，不过肯定是有相应的爬虫技术解决这个问题的，这里基于笔者的知识面暂时只做简单的入门。</p>
<h2 id="1-1-获取URL"><a href="#1-1-获取URL" class="headerlink" title="1.1 获取URL"></a>1.1 获取URL</h2><hr>
<p>以当当购物网 <a target="_blank" rel="noopener" href="http://www.dangdang.com/">http://www.dangdang.com/</a> 为例，浏览器里显示的地址，加上http:&#x2F;&#x2F; 前缀就是我们要设置的URL，有的网站是https:&#x2F;&#x2F; ，这表示http协议的一种加密协议。</p>
<p><img src="https://img-blog.csdnimg.cn/e5aecfb20b4d44c994d1934082148556.png" srcset="/img/loading.gif" lazyload alt="在这里插入图片描述"></p>
<h2 id="1-2-设置请求头"><a href="#1-2-设置请求头" class="headerlink" title="1.2 设置请求头"></a>1.2 设置请求头</h2><hr>
<p>查看请求头 <strong>User-Agent</strong> 通常使用浏览器的调试工具，快捷键 F12</p>
<p>在调试工具中 切换到 <code>[ NetWork ]</code>、中文版的话就是 <code>[ 网络 ] </code></p>
<p>点击后，设置录制状态，并刷新网页，等待一段时间则会显示获取到的网站资源</p>
<p><img src="https://img-blog.csdnimg.cn/26f5bb23926e44f282c241ad17286530.png" srcset="/img/loading.gif" lazyload alt="在这里插入图片描述"></p>
<p>选择 <strong><a target="_blank" rel="noopener" href="http://www.dangdang.com/">www.dangdang.com</a></strong> 这个资源，可以查看底部有一个JSON 格式的键值对 User-Agent : …</p>
<p>其后面的内容就是我们要设置的请求头，这样在向网页发送请求时，网页就会将我们识别成用户，而不是爬虫的机器。<br><img src="https://img-blog.csdnimg.cn/ca6597f3a10b47eaaeeacc1753595336.png" srcset="/img/loading.gif" lazyload alt="在这里插入图片描述"></p>
<h2 id="1-3-python-代码实现"><a href="#1-3-python-代码实现" class="headerlink" title="1.3 python 代码实现"></a>1.3 python 代码实现</h2><hr>
<p>这里的 User-Agent 以笔者的为例，每台机器的请求头不相同，记得更换。</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><code class="hljs python">url = <span class="hljs-string">&#x27;www.dangdang.com&#x27;</span><br><span class="hljs-comment"># 以字典的形式设置请求头</span><br>headers = &#123;<br>    <span class="hljs-string">&#x27;User-Agent&#x27;</span>:  <span class="hljs-string">&#x27;Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.61 Safari/537.36&#x27;</span>,<br>&#125;<br></code></pre></td></tr></table></figure>

<h1 id="爬虫第二步：发送请求，获取响应的网页内容"><a href="#爬虫第二步：发送请求，获取响应的网页内容" class="headerlink" title="爬虫第二步：发送请求，获取响应的网页内容"></a>爬虫第二步：发送请求，获取响应的网页内容</h1><hr>
<p>这一步需要用到 <code>requests库</code> ，其作用是按我们的要求向网站发送请求</p>
<h2 id="2-1-Python-代码实现"><a href="#2-1-Python-代码实现" class="headerlink" title="2.1 Python 代码实现"></a>2.1 Python 代码实现</h2><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre></td><td class="code"><pre><code class="hljs python"><span class="hljs-keyword">import</span> requests<br><br>url = <span class="hljs-string">&#x27;https://category.dangdang.com/pg&#x27;</span><br>headers = &#123;<br>    <span class="hljs-string">&#x27;User-Agent&#x27;</span>:  <span class="hljs-string">&#x27;Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.61 Safari/537.36&#x27;</span>,<br>&#125;<br>response = requests.get(url, headers=headers)<br>html = response.text<br><span class="hljs-built_in">print</span>(html)<br></code></pre></td></tr></table></figure>

<h2 id="2-2-运行结果"><a href="#2-2-运行结果" class="headerlink" title="2.2 运行结果"></a>2.2 运行结果</h2><hr>
<p><img src="https://img-blog.csdnimg.cn/ea9ecabd0c0541609be8ef93e8777f92.png" srcset="/img/loading.gif" lazyload alt="在这里插入图片描述"></p>
<h1 id="爬虫第三步：-解析网页内容，提取数据"><a href="#爬虫第三步：-解析网页内容，提取数据" class="headerlink" title="爬虫第三步： 解析网页内容，提取数据"></a>爬虫第三步： 解析网页内容，提取数据</h1><hr>
<p>通过 requests 发送请求后返回的结果是一个HTML文本，此时它还没有具体的作用。</p>
<p>接下来使用  来自 bs4库的 <strong>BeautifulSoup</strong> ，它支持将HTML内容转化为树状型的数据结构</p>
<p>比如在返回的结果内容中有：</p>
<figure class="highlight html"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><code class="hljs html"><span class="hljs-tag">&lt;<span class="hljs-name">ul</span> <span class="hljs-attr">class</span>=<span class="hljs-string">&quot;ddnewhead_area_list&quot;</span> <span class="hljs-attr">style</span>=<span class="hljs-string">&quot;display: none;&quot;</span> <span class="hljs-attr">id</span>=<span class="hljs-string">&quot;area_list&quot;</span> <span class="hljs-attr">onmouseover</span>=<span class="hljs-string">&quot;this.style.display=&#x27;block&#x27;;&quot;</span></span><br><span class="hljs-tag">         <span class="hljs-attr">onmouseout</span>=<span class="hljs-string">&quot;this.style.display=&#x27;none&#x27;;&quot;</span>&gt;</span><br>         <span class="hljs-tag">&lt;<span class="hljs-name">li</span>&gt;</span><span class="hljs-tag">&lt;<span class="hljs-name">a</span> <span class="hljs-attr">href</span>=<span class="hljs-string">&quot;javascript:void(0);&quot;</span> <span class="hljs-attr">onclick</span>=<span class="hljs-string">&quot;change_area(&#x27;111&#x27;,&#x27;北京&#x27;)&quot;</span> <span class="hljs-attr">num</span>=<span class="hljs-string">&quot;111&quot;</span>&gt;</span>北京<span class="hljs-tag">&lt;/<span class="hljs-name">a</span>&gt;</span><span class="hljs-tag">&lt;/<span class="hljs-name">li</span>&gt;</span><br>         <span class="hljs-tag">&lt;<span class="hljs-name">li</span>&gt;</span><span class="hljs-tag">&lt;<span class="hljs-name">a</span> <span class="hljs-attr">href</span>=<span class="hljs-string">&quot;javascript:void(0);&quot;</span> <span class="hljs-attr">onclick</span>=<span class="hljs-string">&quot;change_area(&#x27;112&#x27;,&#x27;天津&#x27;)&quot;</span> <span class="hljs-attr">num</span>=<span class="hljs-string">&quot;112&quot;</span>&gt;</span>天津<span class="hljs-tag">&lt;/<span class="hljs-name">a</span>&gt;</span><span class="hljs-tag">&lt;/<span class="hljs-name">li</span>&gt;</span><br>         ....<br></code></pre></td></tr></table></figure>

<p>这里是对应网站的这部分内容<br><img src="https://img-blog.csdnimg.cn/b73c0458612b40b78d319d6d9625dcd5.png" srcset="/img/loading.gif" lazyload alt="在这里插入图片描述"></p>
<p>现在可以一次性将里面的省份全部读取下来</p>
<h2 id="3-1-Python-代码实现"><a href="#3-1-Python-代码实现" class="headerlink" title="3.1 Python 代码实现"></a>3.1 Python 代码实现</h2><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre></td><td class="code"><pre><code class="hljs python"><span class="hljs-keyword">import</span> requests<br><span class="hljs-keyword">from</span> bs4 <span class="hljs-keyword">import</span> BeautifulSoup<br>url = <span class="hljs-string">&#x27;https://category.dangdang.com/pg&#x27;</span><br>headers = &#123;<br>    <span class="hljs-string">&#x27;User-Agent&#x27;</span>: <span class="hljs-string">&#x27;Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.61 Safari/537.36&#x27;</span>,<br>&#125;<br>response = requests.get(url, headers=headers)<br>html = response.text<br>soup = BeautifulSoup(html)<br><span class="hljs-keyword">for</span> x <span class="hljs-keyword">in</span> soup.select(<span class="hljs-string">&quot;.ddnewhead_area_list &gt; li &gt; a&quot;</span>):<br>    <span class="hljs-built_in">print</span>(x)<br></code></pre></td></tr></table></figure>

<h2 id="3-2-运行结果"><a href="#3-2-运行结果" class="headerlink" title="3.2 运行结果"></a>3.2 运行结果</h2><p>爬取的结果如下：<br><img src="https://img-blog.csdnimg.cn/8ef29ef326df409a84c806df7a4f6168.png" srcset="/img/loading.gif" lazyload alt="在这里插入图片描述"></p>
<h2 id="3-3-Re库提取关键信息"><a href="#3-3-Re库提取关键信息" class="headerlink" title="3.3 Re库提取关键信息"></a>3.3 Re库提取关键信息</h2><hr>
<p>可以观察到我们需要的信息被 <code>&lt;a&gt; </code> 标签所包裹，现在则可以使用简单的正则表达式将其获取</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><code class="hljs python"><span class="hljs-built_in">print</span>(re.search(<span class="hljs-string">&#x27;&gt;([^&lt;]+)&lt;&#x27;</span>, <span class="hljs-built_in">str</span>(x)).group(<span class="hljs-number">1</span>))<br></code></pre></td></tr></table></figure>
<p><strong>实现效果</strong><br><img src="https://img-blog.csdnimg.cn/528ff4cb9acc4d1f826de42b8e1d233b.png" srcset="/img/loading.gif" lazyload alt="在这里插入图片描述"></p>
<h1 id="爬虫第四步：使用DataFrame保存数据"><a href="#爬虫第四步：使用DataFrame保存数据" class="headerlink" title="爬虫第四步：使用DataFrame保存数据"></a>爬虫第四步：使用DataFrame保存数据</h1><hr>
<p>DataFrame类似一个Excel表格，而且有许多原生的API进行数据变换的操作，这里主要用来保存。</p>
<h2 id="4-1-根据列表创建-DataFrame对象"><a href="#4-1-根据列表创建-DataFrame对象" class="headerlink" title="4.1 根据列表创建 DataFrame对象"></a>4.1 根据列表创建 DataFrame对象</h2><hr>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br></pre></td><td class="code"><pre><code class="hljs python"><span class="hljs-keyword">import</span> requests<br><span class="hljs-keyword">from</span> bs4 <span class="hljs-keyword">import</span> BeautifulSoup<br><span class="hljs-keyword">import</span> re<br><span class="hljs-keyword">import</span> pandas <span class="hljs-keyword">as</span> pd<br>url = <span class="hljs-string">&#x27;https://category.dangdang.com/pg&#x27;</span><br>headers = &#123;<br>    <span class="hljs-string">&#x27;User-Agent&#x27;</span>:  <span class="hljs-string">&#x27;Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.61 Safari/537.36&#x27;</span>,<br>&#125;<br>response = requests.get(url, headers=headers)<br>html = response.text<br>soup = BeautifulSoup(html)<br>data = []<br><span class="hljs-keyword">for</span> x <span class="hljs-keyword">in</span> soup.select(<span class="hljs-string">&quot;.ddnewhead_area_list &gt; li &gt; a&quot;</span>):<br>    data.append(re.search(<span class="hljs-string">&#x27;&gt;([^&lt;]+)&lt;&#x27;</span>, <span class="hljs-built_in">str</span>(x)).group(<span class="hljs-number">1</span>))<br>df = pd.DataFrame(&#123;<span class="hljs-string">&#x27;中国省份&#x27;</span> : data&#125;)<br><span class="hljs-built_in">print</span>(df)<br></code></pre></td></tr></table></figure>

<p><strong>运行结果：</strong></p>
<p><img src="https://img-blog.csdnimg.cn/eaf5a7d1369c4e2fac300dcd6de59b10.png" srcset="/img/loading.gif" lazyload alt="在这里插入图片描述"></p>
<h2 id="4-2-导出DataFrame到本地"><a href="#4-2-导出DataFrame到本地" class="headerlink" title="4.2 导出DataFrame到本地"></a>4.2 导出DataFrame到本地</h2><hr>
<p>导出DataFrame通过调用该对象的 <strong>to_xxx</strong> 方法，其中有常用的几个参数：</p>
<ul>
<li>sep 设置列之间的分隔符号</li>
<li>index 设置导出的结果文件是否带行下标</li>
<li>header 设置导出的结果文件是否带列名<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><code class="hljs python">df.to_csv(<span class="hljs-string">&#x27;test.txt&#x27;</span>, sep=<span class="hljs-string">&#x27;\t&#x27;</span>,index=<span class="hljs-literal">False</span>, header=<span class="hljs-literal">None</span>)<br></code></pre></td></tr></table></figure>
<img src="https://img-blog.csdnimg.cn/b164162647c447a68daf03cb3736e187.png" srcset="/img/loading.gif" lazyload alt="在这里插入图片描述"><br>至此，已实现了最入门的python爬虫过程。</li>
</ul>

                
              </div>
            
            <hr/>
            <div>
              <div class="post-metas my-3">
  
    <div class="post-meta mr-3 d-flex align-items-center">
      <i class="iconfont icon-category"></i>
      

<span class="category-chains">
  
  
    
      <span class="category-chain">
        
  <a href="/categories/Python/" class="category-chain-item">Python</a>
  
  
    <span>></span>
    
  <a href="/categories/Python/%E7%88%AC%E8%99%AB/" class="category-chain-item">爬虫</a>
  
  

  

      </span>
    
  
</span>

    </div>
  
  
    <div class="post-meta">
      <i class="iconfont icon-tags"></i>
      
        <a href="/tags/Python/">#Python</a>
      
        <a href="/tags/%E7%88%AC%E8%99%AB/">#爬虫</a>
      
    </div>
  
</div>


              
  

  <div class="license-box my-3">
    <div class="license-title">
      <div>Python爬虫 笔记1 _ 爬虫案例及模板 简要四步爬取购物网站信息 _ 基于 requests、re、 pandas、 bs4</div>
      <div>http://example.com/2021/12/08/Python爬虫 笔记1 _ 爬虫案例及模板 简要四步爬取购物网站信息 _ 基于 requests、re、 pandas、 bs4/</div>
    </div>
    <div class="license-meta">
      
        <div class="license-meta-item">
          <div>作者</div>
          <div>John Doe</div>
        </div>
      
      
        <div class="license-meta-item license-meta-date">
          <div>发布于</div>
          <div>2021年12月8日</div>
        </div>
      
      
      <div class="license-meta-item">
        <div>许可协议</div>
        <div>
          
            
            
              <a target="_blank" href="https://creativecommons.org/licenses/by/4.0/">
              <span class="hint--top hint--rounded" aria-label="BY - 署名">
                <i class="iconfont icon-by"></i>
              </span>
              </a>
            
          
        </div>
      </div>
    </div>
    <div class="license-icon iconfont"></div>
  </div>



              
                <div class="post-prevnext my-3">
                  <article class="post-prev col-6">
                    
                    
                      <a href="/2021/12/14/%E5%A4%A7%E6%95%B0%E6%8D%AE%E5%B7%A5%E7%A8%8B%E5%AE%9E%E8%B7%B5v1.0%E7%89%88%E6%9C%AC%20_%20%EF%BC%88%E4%B8%80%EF%BC%89%E7%8E%AF%E5%A2%83%E6%90%AD%E5%BB%BA%20(1)%20_%20%E7%BB%9F%E4%B8%80%E9%85%8D%E7%BD%AEhadoop3.1.3%E5%AE%8C%E5%85%A8%E5%88%86%E5%B8%83%E5%BC%8F%E9%9B%86%E7%BE%A4%E4%B8%89%E4%B8%AA%E8%8A%82%E7%82%B9/" title="大数据工程实践v1.0版本 _ （一）环境搭建 (1) _ 统一配置hadoop3.1.3完全分布式集群三个节点">
                        <i class="iconfont icon-arrowleft"></i>
                        <span class="hidden-mobile">大数据工程实践v1.0版本 _ （一）环境搭建 (1) _ 统一配置hadoop3.1.3完全分布式集群三个节点</span>
                        <span class="visible-mobile">上一篇</span>
                      </a>
                    
                  </article>
                  <article class="post-next col-6">
                    
                    
                      <a href="/2021/12/06/BUG%E8%A7%A3%E5%86%B3%20_%20Hadoop%E9%9B%86%E7%BE%A4%E4%B8%8B%E5%90%AF%E5%8A%A8Spark%20On%20Yarn%20%E6%8A%A5%E9%94%99%E9%97%AE%E9%A2%98%E5%8F%8A%E8%A7%A3%E5%86%B3%20%E7%89%88%E6%9C%AC%E5%85%BC%E5%AE%B9%E9%97%AE%E9%A2%98%20Hadoop3.1.3%20%E4%B8%8ESpark3.0.0/" title="BUG解决 _ Hadoop集群下启动Spark On Yarn 报错问题及解决 版本兼容问题 Hadoop3.1.3 与Spark3.0.0">
                        <span class="hidden-mobile">BUG解决 _ Hadoop集群下启动Spark On Yarn 报错问题及解决 版本兼容问题 Hadoop3.1.3 与Spark3.0.0</span>
                        <span class="visible-mobile">下一篇</span>
                        <i class="iconfont icon-arrowright"></i>
                      </a>
                    
                  </article>
                </div>
              
            </div>

            
          </article>
        </div>
      </div>
    </div>

    <div class="side-col d-none d-lg-block col-lg-2">
      
  <aside class="sidebar" style="margin-left: -1rem">
    <div id="toc">
  <p class="toc-header"><i class="iconfont icon-list"></i>&nbsp;目录</p>
  <div class="toc-body" id="toc-body"></div>
</div>



  </aside>


    </div>
  </div>
</div>





  



  



  



  



  







    

    
      <a id="scroll-top-button" aria-label="TOP" href="#" role="button">
        <i class="iconfont icon-arrowup" aria-hidden="true"></i>
      </a>
    

    
      <div class="modal fade" id="modalSearch" tabindex="-1" role="dialog" aria-labelledby="ModalLabel"
     aria-hidden="true">
  <div class="modal-dialog modal-dialog-scrollable modal-lg" role="document">
    <div class="modal-content">
      <div class="modal-header text-center">
        <h4 class="modal-title w-100 font-weight-bold">搜索</h4>
        <button type="button" id="local-search-close" class="close" data-dismiss="modal" aria-label="Close">
          <span aria-hidden="true">&times;</span>
        </button>
      </div>
      <div class="modal-body mx-3">
        <div class="md-form mb-5">
          <input type="text" id="local-search-input" class="form-control validate">
          <label data-error="x" data-success="v" for="local-search-input">关键词</label>
        </div>
        <div class="list-group" id="local-search-result"></div>
      </div>
    </div>
  </div>
</div>

    

    
  </main>

  <footer>
    <div class="footer-inner">
  
    <div class="footer-content">
       <a href="https://hexo.io" target="_blank" rel="nofollow noopener"><span>Hexo</span></a> <i class="iconfont icon-love"></i> <a href="https://github.com/fluid-dev/hexo-theme-fluid" target="_blank" rel="nofollow noopener"><span>Fluid</span></a> 
    </div>
  
  
  
  
</div>

  </footer>

  <!-- Scripts -->
  
  <script  src="https://lib.baomitu.com/nprogress/0.2.0/nprogress.min.js" ></script>
  <link  rel="stylesheet" href="https://lib.baomitu.com/nprogress/0.2.0/nprogress.min.css" />

  <script>
    NProgress.configure({"showSpinner":false,"trickleSpeed":100})
    NProgress.start()
    window.addEventListener('load', function() {
      NProgress.done();
    })
  </script>


<script  src="https://lib.baomitu.com/jquery/3.6.0/jquery.min.js" ></script>
<script  src="https://lib.baomitu.com/twitter-bootstrap/4.6.1/js/bootstrap.min.js" ></script>
<script  src="/js/events.js" ></script>
<script  src="/js/plugins.js" ></script>


  <script  src="https://lib.baomitu.com/typed.js/2.0.12/typed.min.js" ></script>
  <script>
    (function (window, document) {
      var typing = Fluid.plugins.typing;
      var subtitle = document.getElementById('subtitle');
      if (!subtitle || !typing) {
        return;
      }
      var text = subtitle.getAttribute('data-typed-text');
      
        typing(text);
      
    })(window, document);
  </script>




  
    <script  src="/js/img-lazyload.js" ></script>
  




  
<script>
  Fluid.utils.createScript('https://lib.baomitu.com/tocbot/4.18.2/tocbot.min.js', function() {
    var toc = jQuery('#toc');
    if (toc.length === 0 || !window.tocbot) { return; }
    var boardCtn = jQuery('#board-ctn');
    var boardTop = boardCtn.offset().top;

    window.tocbot.init({
      tocSelector     : '#toc-body',
      contentSelector : '.markdown-body',
      headingSelector : CONFIG.toc.headingSelector || 'h1,h2,h3,h4,h5,h6',
      linkClass       : 'tocbot-link',
      activeLinkClass : 'tocbot-active-link',
      listClass       : 'tocbot-list',
      isCollapsedClass: 'tocbot-is-collapsed',
      collapsibleClass: 'tocbot-is-collapsible',
      collapseDepth   : CONFIG.toc.collapseDepth || 0,
      scrollSmooth    : true,
      headingsOffset  : -boardTop
    });
    if (toc.find('.toc-list-item').length > 0) {
      toc.css('visibility', 'visible');
    }
  });
</script>


  <script src=https://lib.baomitu.com/clipboard.js/2.0.10/clipboard.min.js></script>

  <script>Fluid.plugins.codeWidget();</script>


  
<script>
  Fluid.utils.createScript('https://lib.baomitu.com/anchor-js/4.3.1/anchor.min.js', function() {
    window.anchors.options = {
      placement: CONFIG.anchorjs.placement,
      visible  : CONFIG.anchorjs.visible
    };
    if (CONFIG.anchorjs.icon) {
      window.anchors.options.icon = CONFIG.anchorjs.icon;
    }
    var el = (CONFIG.anchorjs.element || 'h1,h2,h3,h4,h5,h6').split(',');
    var res = [];
    for (var item of el) {
      res.push('.markdown-body > ' + item.trim());
    }
    if (CONFIG.anchorjs.placement === 'left') {
      window.anchors.options.class = 'anchorjs-link-left';
    }
    window.anchors.add(res.join(', '));
  });
</script>


  
<script>
  Fluid.utils.createScript('https://lib.baomitu.com/fancybox/3.5.7/jquery.fancybox.min.js', function() {
    Fluid.plugins.fancyBox();
  });
</script>


  <script>Fluid.plugins.imageCaption();</script>

  <script  src="/js/local-search.js" ></script>





<!-- 主题的启动项，将它保持在最底部 -->
<!-- the boot of the theme, keep it at the bottom -->
<script  src="/js/boot.js" ></script>


  

  <noscript>
    <div class="noscript-warning">博客在允许 JavaScript 运行的环境下浏览效果更佳</div>
  </noscript>
</body>
</html>
